
from sklearn.preprocessing import LabelEncoder
import tensorflow.contrib.keras as kr
import time
import warnings
from sqlalchemy import create_engine
import psycopg2
import pandas as pd
# 把9分类.xls已有的库进行分词
# 连接数据库

# 获得数据
# 连接数据库
conn = psycopg2.connect(database='ltzd', user='ai',
                        password='ai001', host='127.0.0.1', port='5432')


# 编写Sql，只取前两行数据
sql = 'select index,"CATALOG_目录",keyword from ai_9fenlei_data'

# 获得数据
df = pd.read_sql_query(sql, conn, index_col='index')

print(df)
#df.reset_index(level=0, inplace=True)

# print(pd_data)

# 关闭指针和数据库

conn.close()


warnings.filterwarnings('ignore')
startTime = time.time()


def printUsedTime():
    used_time = time.time() - startTime
    print('used time: %.2f seconds' % used_time)


with open('E:\人工智能大作业\九分类.txt', encoding='utf8') as file:
    line_list = [k.strip() for k in file.readlines()]
    train_label_list = [k.split()[0] for k in line_list]
    train_content_list = [k.split(maxsplit=1,)[1] for k in line_list]

with open('E:\E:\人工智能大作业\out.txt', encoding='utf8') as file:
    vocabulary_list = [k.strip() for k in file.readlines()]

print('0.load train data finished')
printUsedTime()
word2id_dict = dict([(b, a) for a, b in enumerate(vocabulary_list)])
# print(word2id_dict)


def content2idList(content): return [word2id_dict[word]
                                     for word in content if word in word2id_dict]


train_idlist_list = [content2idList(content) for content in train_content_list]
# print(train_idlist_list[0])
vocabolary_size = 5000  # 词汇表达小
sequence_length = 150  # 序列长度
embedding_size = 64  # 词向量大小
num_hidden_units = 256  # LSTM细胞隐藏层大小
num_fc1_units = 64  # 第1个全连接下一层的大小
dropout_keep_probability = 0.5  # dropout保留比例
num_classes = 91  # 类别数量
learning_rate = 1e-3  # 学习率
batch_size = 50   # 每批训练大小

train_X = kr.preprocessing.sequence.pad_sequences(
    train_idlist_list, sequence_length)
print(train_X)
labelEncoder = LabelEncoder()
train_y = labelEncoder.fit_transform(train_label_list)
print(train_y)
train_Y = kr.utils.to_categorical(train_y, num_classes)
print(train_Y)
